package com.summba.corpus.extractor;

import java.io.BufferedReader;
import java.io.File;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;

import com.summba.common.Common;

public class WeiboExtractor {
	private String dbFilePath;
	private Set<String> contentSet = new HashSet<String>();
	private Set<String> keywordContentSet = new HashSet<String>();
	private List<String> corpusFileList = new ArrayList<String>();
	
	public WeiboExtractor( String corpusFilePath , String saveFile ) throws Exception{
		dbFilePath = saveFile;
		File tempFile = new File(dbFilePath);
		if( !tempFile.exists() ){
			tempFile.createNewFile();
		}
		
		corpusFileList.addAll(Common.getFloderAllFileList(corpusFilePath, true));
		/*
		 * load corpus content db file
		 */
		BufferedReader dbFileTempBR = Common.getFileInfo(dbFilePath);
		List<String> tempList =  Common.bufferedReaderToList(dbFileTempBR);	
		System.out.println( "Empty : " + tempList.isEmpty() );
		contentSet.addAll(tempList);
	}
	
	public void getTextContent(){
		for (Iterator<String> iterator = corpusFileList.iterator(); iterator.hasNext();) {
			Set<String> tempSet = new HashSet<String>();
			String tempKeyString = "";
			String filePath = (String) iterator.next();
			System.out.println("==================================");
			System.out.println("filePath: "+filePath);
			System.out.println("==================================");
			BufferedReader tempBR = Common.getFileInfo(filePath);
			List<String> tempList = Common.bufferedReaderToList(tempBR);
			for (Iterator<String> iteratorContent = tempList.iterator(); iteratorContent.hasNext();) {
				String string = (String) iteratorContent.next();
				if(	string.indexOf("keyword:",0) != -1 ){
					tempKeyString = string;
				}
				if( string.indexOf("text:",0) != -1 ){
					if( !contentSet.contains(string) ){
						tempKeyString = tempKeyString + string;
						tempSet.add(string);
						keywordContentSet.add(tempKeyString);
						tempKeyString = "";
					}
				}
			}
			System.out.println("Write the DB File");
			writeToDBFile(dbFilePath, keywordContentSet);
			contentSet.addAll(tempSet);
			keywordContentSet.clear();
		}
		System.out.println("finish!!");
	}
	
	/*
	public void closeDBFile(){
		System.out.println("Write the DB File");
		writeToDBFile(dbFilePath,tempSet);
		System.out.println("finish!!");
	}
	*/
	
	private void writeToDBFile(String FilePath , Set<String> allKeyWord){
		for (Iterator<String> iterator = allKeyWord.iterator(); iterator.hasNext();) {
			String type =  (String) iterator.next();
			Common.appendFile(FilePath, type+"\n");
		}
	}
}
